BOT_NAME = 'lago'
SPIDER_MODULES = ['lago.spiders']
NEWSPIDER_MODULE = 'lago.spiders'
#  USER_AGENT = 'lago (Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/63.0.3239.84 Chrome/63.0.3239.84 Safari/537.36)'
USER_AGENT = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)"

HTTPERROR_ALLOWED_CODES = [301, 302]
CONCURRENT_REQUESTS  =  100
DOWNLOAD_TIMEOUT  =  15

DOWNLOADER_MIDDLEWARES = {
    'scrapy.downloadermiddleware.useragent.UserAgentMiddleware': None, 
    'lago.middlewares.MyUserAgentMiddleware': 400,
    'lago.middlewares.RandomUserAgentMiddleware': 343,
    'lago.middlewares.ProxyMiddleware':544

        }
ROBOTSTXT_OBEY = False

COOKIES_ENABLES = False

DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter"
SCHEDULER = "scrapy_redis.scheduler.Scheduler"
SCHEDULER_PERSIST = True
ITEM_PIPELINES = {
        'lago.pipelines.TextPipeline': 300,
        #  'lago.pipelines.MysqlPipeline': 301,
        }

LOG_LEVEL = 'DEBUG'
DOWNLOAD_DELAY = 10

DB_USER = 'feiza1'
DB_PASSWD = 'feiza1'
DB_DB = 'lago'
DB_HOST = 'localhost'
DB_PORT = 3306
DB_CHARSET = 'utf8'
